*This program does a basic cleanup on the student-level SCS files
*DESE provides files in SPSS format and they must be converted to Stata

clear
set more off

local raw "$raw\SCS"
local saves "$raw\saves"
cd "`raw'"

// set the locals
// if importing one new year of data, set y_start and y_end to be the same
local import = 0 // set equal to 1 only if bringing in new data
local y_start = 11 // year that you want to start data import
local y_end = 23 // last year for data import

local y_last = 23 // last year of data available and imported

// Convert dta from spss to dta
if `import' {

    // Loop through years and terms in the SCS data
    // Files appended with A are all of the courses enrolled in by October
    // Files appended with D are all of the courses enrolled in by end of the year
    // We only want to import the end of year files
    forval y = `y_start'/`y_end' {
        foreach term in D {
            // Import each SPSS file
            if `y' == 16 {
                import spss using "`raw'\SCS`y'`term'- Incomplete", case(lower) clear
            }
            else if `y' == 17 & "`term'" == "D" {
                import spss using "`raw'\SCS`y'`term' Incomplete", case(lower) clear
            }
            else if (`y' == 20 | `y' == 21 | `y' == 22) & "`term'" == "A" {
                disp "No data on the Fall term for 20`y'"
            }
            else {
                import spss using "`raw'\SCS`y'`term'", case(lower) clear
            }
            
            // save the imported files to the raw folder
            // A indicates the first semester in an academic year
            if "`term'" == "A" & !(`y' == 20 | `y' == 21 | `y' == 22) {
                gen year = 20`y'
                gen sem = 1
                save "`raw'\stata raw\scs`y'_1.dta", replace
            }
            else if "`term'" == "D" {
                gen year = 20`y'
                gen sem = 2
                save "`raw'\stata raw\scs`y'_2.dta", replace
            }
        }
    }
}

// Append the SCS files together
clear
// Loop through years and terms in the SCS data
// Only append together the end of year files, labeled with _2 in filename
forval y = 11/`y_last' {
        append using "`raw'\stata raw\scs`y'_2.dta"
}

// save the appended data
save "`saves'\scs_appended.dta", replace

// destring sasid to be able to merge with the lottery sample
destring sasid, replace
format sasid %13.0f
save "${data_clean}\course_data.dta", replace

